# -*- coding:utf-8 -*-

import requests
from lxml import etree
import redis
import re

class Agent:

    agentUrl =  'http://www.youdaili.net/Daili/http/'
    agentSet = set()
    headers = {
            'Host': 'www.youdaili.net',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
            'Accept-Encoding': 'gzip, deflate',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
    }

    def __init__(self):
        firstList = self.getFirstListUrl()
        if not firstList:
            raise Exception('代理页面获取失败')
        self.setAgents(firstList)
        
    def setRedis(self):
        redisClient = redis.Redis(host='127.0.0.1',port=6379)
        for agent in self.agentSet:
            redisClient.sadd('http-agent-set',agent)
        
    def newAgents(self):
        for agent in self.agentSet:
            yield agent

    @classmethod
    def setAgents(cls,url,recursive=False):
        r = requests.get(url,headers=cls.headers)
        selector = etree.HTML(r.content)
        agentsString = selector.xpath("//div[@class='content']/p/text()")
        for agentString in agentsString:
            try:
                agent = re.compile(r'\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}:\d{2,5}@(http|HTTP|https|HTTPS)').match(agentString).group()
                cls.agentSet.add(agent)
            except Exception,e:
                pass
                
        if not recursive:
            pageurls = set()
            for url in selector.xpath("//div[@class='pagebreak']/li/a/@href"):
                if url and url != '#':
                    pageurls.add(url)
            for url in pageurls:
                cls.setAgents('http://www.youdaili.net/Daili/http/%s' % url,True)
        
        
    @classmethod
    def getFirstListUrl(cls):
        r = requests.get(cls.agentUrl,headers=cls.headers)
        selector = etree.HTML(r.content)
        urls = selector.xpath("//div[@class='chunlist']/ul/li[1]//a/@href")
        return urls[0] if urls else False
        
        
        
        
if __name__ == '__main__':

    Agent().setRedis()


    # Agent.setAgents('http://www.youdaili.net/Daili/http/35615.html')
    #content =  r.content.decode("ISO-8859-2").encode("utf-8")
    """
    抓取的 实体 如&nbsp;不能正常解析成gbk 会报错
    参考
    http://www.aobosir.com/blog/2016/11/26/python3-UnicodeEncodeError-gbk-codec-can%27t-encode-character-xa0/
    http://stackoverflow.com/questions/10993612/python-removing-xa0-from-string
    """
    ## print agentString.replace(u'\xc2\xa0 ', u' '),
    # print agentString.strip(),
    # re.sub(r'[^\x00-\x7F]+','',agentString).decode('utf-8','ignore').strip()
    # print u'````' ,
        
    